import urllib.request as ur
import lxml.etree as le
import user_agent

word = input("请输入要查找的关键字：")
start_page = int(input("请输入开始页码："))
end_page = int(input("请输入结束页码："))

def getRequest(url):
    return ur.Request(
        url=url,
        headers={
            'User-Agent': user_agent.get_user_agent_pc(),
            'Cookie': 'acw_tc=2760829615741768974065019e8879589ace9f49d6852b77fc884305dae16b; acw_sc__v2=5dd40881c75d1d0502af4b9da75d41588f818462; acw_sc__v3=5dd40881a117a41997f25af72488823ff56723bf; uuid_tt_dd=10_18749719900-1574176897804-107579; dc_session_id=10_1574176897804.531143; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1574176899; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_18749719900-1574176897804-107579; c-login-auto=1; announcement=%257B%2522isLogin%2522%253Afalse%252C%2522announcementUrl%2522%253A%2522https%253A%252F%252Fblogdev.blog.csdn.net%252Farticle%252Fdetails%252F103053996%2522%252C%2522announcementCount%2522%253A0%252C%2522announcementExpire%2522%253A3600000%257D; dc_tos=q182oc; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1574176924'
        }
    )

def getProxyOpener():
    proxy_address = ur.urlopen('http://api.ip.data5u.com/dynamic/get.html?order=f65ebccd58d38538fe5adb74d246b1f4&sep=4').read().decode('utf-8').strip()

    proxy_handler = ur.ProxyHandler(
        {
            'http': proxy_address
        }
    )
    return  ur.build_opener(proxy_handler)

for pn in range(start_page,end_page+1):
    url = 'https://so.csdn.net/so/search/s.do?p=%s&q=%s&t=blog&viparticle=&domain=&o=&s=&u=&l=&f=&rbg=0' % (pn, word)
    print(url)
    request = getRequest(url)
    try:
        response = getProxyOpener().open(request).read()
        href_s = le.HTML(response).xpath('//span[@class="down fr"]/../span[@class="link"]/a/@href')
        print(href_s)
        for href in href_s:
            try:
                response_blog = getProxyOpener().open(getRequest(href)).read()
                # print(getProxyOpener().open(getRequest(href)).read().decode('utf-8'))
                title = le.HTML(response_blog).xpath('//h1[@class="title-article"]/text()')[0]
                # title = le.HTML(response_blog).xpath('//div[@class="article-title-box"]/h1')
                print(title)
                with open('blog/%s.html' % title, 'wb') as f:
                    f.write(response_blog)
            except Exception as err:
                print(err)
    except:
        pass